package com.asiainfo.huxiu;

import cn.edu.hfut.dmic.contentextractor.ContentExtractor;
import cn.edu.hfut.dmic.contentextractor.News;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.HttpURLConnection;
import java.net.URL;

/**
 * HuXiuIndex
 * 虎嗅首页:https://www.huxiu.com/
 *
 * @Author jhr
 * @Date 2022/4/1
 */
public class HuXiuIndex extends BreadthCrawler {
    private static final Logger logger = LoggerFactory.getLogger(HuXiuIndex.class);
    private static final String crawlPath = "/Users/jhr/data/db/huxiuIndex";
    //    private  static String crawlPath = "";
//    虎嗅网首页：https://www.huxiu.com/
    public static String seed = "";
    //    查找匹配的网站
//  https://www.huxiu.com/article/.*.html  匹配类型
    public static String RegularUrl = "";
//    static JdbcTemplate jdbcTemplate = null;

    public HuXiuIndex(String Seed, String regularUrl) {
        super(crawlPath, false);
        seed = Seed;
        RegularUrl = regularUrl;
//        crawlPath = CrawlPath;
        CrawlDatum crawlDatum = new CrawlDatum(seed).meta("depth", "2");
        addSeed(crawlDatum);
        this.addRegex(RegularUrl);
        setThreads(2);
    }

    @Override
    public void visit(Page page, CrawlDatums next) {
        try {
            URL Url = new URL(page.url());
            HttpURLConnection conn = (HttpURLConnection) Url.openConnection();
            conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
        } catch (Exception e) {
            e.printStackTrace();
        }
        if (page.matchUrl(RegularUrl)) {
//            System.out.println(page.url());
            News news = null;
            try {
                news = ContentExtractor.getNewsByUrl(page.url());
            } catch (Exception e) {
                e.printStackTrace();
            }
            assert news != null;
            System.out.println("爬取网址：" + news.getUrl());
            String releaseTime = page.select("span.article__time").text();
            System.out.println("发布时间：" + releaseTime);
//            System.out.println("发布时间：" + news.getTime());
            System.out.println("文章标题：" + news.getTitle());
            System.out.println("文章内容：" + news.getContent());
            String author = page.select("span.author-info__username").text();
            System.out.println("作者:" + author);
//            无阅读量
            String source = "虎嗅网";
            System.out.println("来源：" + source);
//            System.out.println("contentElement"+contentElement);
            /*
             * news.getContentElement() 返回的是正文所在的标签元素
             */
            /* 根据标签名递归查询正文下的图片标签
             * 同理可以获取正文标签下其它任意想要获取的内容*/
//            Elements elements = contentElement.getElementsByTag("img");
        } else {
            Elements elements = page.select("a");
            System.out.println("获取到链接个数：" + elements.size());
            for (Element element : elements) {
                logger.debug("获取到匹配的url=" + element.attr("abs:href"));
                String regRx = RegularUrl;
//                匹配符合正则的网站
                if (element.attr("abs:href").matches(regRx)) {
                    CrawlDatum meta = new CrawlDatum(element.attr("abs:href")).meta("depth", "1").meta("refer", seed);
                    next.add(meta);
                } else {
                    System.out.println("正则URL不匹配！！！");
                }
            }
        }
        return ;
    }

    public static void main(String[] args) throws Exception {
//        虎嗅首页
//        String seed = "https://www.huxiu.com/";
//        财经板块首页
//        String seed = "https://www.huxiu.com/channel/115.html";

        String seed = "https://www.huxiu.com/channel/115.html";
        String regular = "https://www.huxiu.com/article/.*.html";
        HuXiuIndex huXiuIndex = new HuXiuIndex(seed, regular);
        huXiuIndex.start(2);
    }
}
